package mblog.web.job;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.UUID;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

public class Credit52 {
	static String tags = "信用卡,小额贷,赚小贝,瑞银信,VIP会员,催收,POS,优惠活动,征信,积分,借呗,花呗,白条,芝麻信用,工行,招行,中行,建行,交行,中信,广发,光大,浦发,民生,兴业,农行,平安,申卡,提额,销卡,苏宁,任性付";
	static Map<String, String> cookies = null;
	static String userAgent = null;
	static String MBLOG_JSESSIONID = null;
	static String newPicDir = "";
	static String tysSpiderLogDir = "D:/tysSpiderLog/";
	{
		/************************* CONFIG *********************/

		MBLOG_JSESSIONID = "61973D76BCF24C681EBBAD3261918BE2";// MBLOG SESSION

		// credit 51 session
		cookies = new HashMap<>();
		// cookies.put("JSESSIONID", "");

//		userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0";
		userAgent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36";
		// PIC SERVER
		//newPicDir = "G:/apache-tomcat-8.0.32/webapps/examples/xb_pic/";
		newPicDir = "E:/ZHIYANWORKSPACE2.0/.metadata/.plugins/org.eclipse.wst.server.core/tmp3/wtpwebapps/mblog-web/store/jianshu_pic/";

		/************************* CONFIG *********************/

	}

	//当前TAG
	static String[] cur_tag = new String[]{"信用卡","平安"};
	@Test
	public void processFace() {
//		 String url ="http://bbs.51credit.com/forum.php?mod=forumdisplay&fid=3&filter=heat&orderby=heats";//工行
		String url ="http://bbs.51credit.com/forum.php?mod=forumdisplay&fid=115&filter=heat&orderby=heats";//招商
		//String url ="http://www.jianshu.com/";//招商
//		String url = "http://bbs.51credit.com/forum.php?mod=viewthread&tid=1037839&extra=page%3D1%26orderby%3Dheats";
		String tid = getArcticleId(url);
		if ("-1".equals(tid)) {
			processMutil(url);
		} else {
			processOne(url);
		}
		System.out.println("over------------------------------------");
	}

	void processMutil(String url) {
		try {
			Document doc = getDoc(url);
			// System.out.println(doc.body());
			Elements links = doc.body().select("th.new");
			for (Element link : links) {
				link.select("em").remove();
				Elements alinks = link.select("a");
				// System.out.println(alinks.get(0));
				String alink = alinks.get(0).absUrl("href");
				System.out.println("开始抓取：" + alink);
				processOne(alink);
			}

		} catch (Exception e) {
		}

	}

	void processOne(String url) {
		try {
			String tid = getArcticleId(url);
			if ("-1".equals(tid)) {
				System.out.println("不能识别TID：" + url);
				return;
			}
			try {
				Thread.sleep((long)new Random().nextInt(5000));
			} catch (Exception e) {
			}
			 /*if (!isNewArticle(tid)) { System.out.println("文章已抓取过：" + url);
			 return; }
			 */

			Document doc = getDoc(url);
			doc.setBaseUri(url);//
			Map<String, String> article = parseDoc(doc);
			if (article == null) {
				return;
			}
			postRsltMap(article);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	/**
	 * 发送到mblog
	 * 
	 * @param artcleMap
	 * @throws IOException
	 */
	void postRsltMap(Map<String, String> artcleMap) {
		if (null == (artcleMap.get("content"))) {
			artcleMap.get("errorUrl");
			return;
		}

		MblogPost.post(artcleMap);
	}

	/**
	 * 获取DOCUMENT对象
	 * 
	 * @param url
	 * @return
	 * @throws IOException
	 */
	Document getDoc(String url) throws IOException {
		Response resp = Jsoup.connect(url).userAgent(userAgent).cookies(cookies).execute();

		cookies = resp.cookies();
//		 System.out.println("访问："+url);
		System.out.println("返回：" + cookies);
		Document doc = resp.parse();
		return doc;
	}

	/***
	 * 解析DOCUMENT
	 * 
	 * @param doc
	 * @return
	 * @throws IOException
	 */
	Map<String, String> parseDoc(Document doc)   {
		Map<String, String> artcleMap = new HashMap<>();
		try{

			artcleMap.put("srcUrl", ""+doc.baseUri());
			artcleMap.put("title", doc.title());

			Element bodyObj = doc.body();
			// System.out.println(bodyObj);
			// 提取各楼帖子文章
			Element allFloor = bodyObj.select("div#postlist").get(0);// 所有楼层
			allFloor.select("script").remove();//去除脚本
			// System.out.println(allFloor);
			Elements titleObjs = allFloor.select("h1");
			if(titleObjs!=null &&titleObjs.size()>0){
				artcleMap.put("title",titleObjs.get(0).text());	
			}
			
			Element fristFloor = allFloor.select("div>div").get(0);// 第一层楼
			// System.out.println(fristFloor);
			String hid = fristFloor.attr("id").replaceAll("post_", "");

			Elements editTimeObjs = fristFloor.select("em#authorposton" + hid);//发帖时间
			String editTime = editTimeObjs.get(0).text(); 
			editTime = editTime.substring(editTime.length() - 18, editTime.length() - 3) + ":00";
			artcleMap.put("summary", editTime);

			
			
			// 替换所有超链接为空事件
			Element articleContent = fristFloor.select("td#postmessage_" + hid).get(0);
			if (articleContent.text().contains("查看本帖隐藏内容")) {
				throw new RuntimeException("有隐藏内容，手动添加吧");
			}

			articleContent.select("a[href]").attr("href", "javascript:void()");
			// 替换图片
			Elements imgObjs = articleContent.select("img[src]");
			for (Element img : imgObjs) {
				
				String img_abs="http://bbs.51credit.com/"+img.attr("zoomfile");
				if("http://bbs.51credit.com/".equals(img_abs)){
					continue;
				}
				img.attr("src", getNewImgUrl(img_abs));
			}

			artcleMap.put("content", articleContent.toString());
			artcleMap.put("tags", getTag(articleContent.toString()));

			 saveHtmFile(articleContent.toString());
//			System.out.println(articleContent);
		}catch(Exception e){
			artcleMap.put("content", "管理员"+doc.baseUri());
			artcleMap.put("tags", "管理员");
			return artcleMap;
		}

		return artcleMap;
	}

	/***
	 * 解析DOCUMENT
	 * 
	 * @param doc
	 * @return
	 * @throws IOException
	 */
	Map<String, String> parseDoc2(Document doc) throws IOException {
		Map<String, String> artcleMap = new HashMap<>();
		artcleMap.put("title", doc.title());

		Element bodyObj = doc.body();
		System.out.println(bodyObj);
		// 提取各楼帖子文章
		Elements articleContents = bodyObj.select("div.message");
		Elements articleContent = articleContents.eq(0);

		// 移除 本楼【最后编辑状态】
		Elements editObjs = articleContent.select("i");
		String editTime = editObjs.eq(0).text();// 本帖最后由 hu861884666 于 2014-7-6
												// 13:18 编辑
		if (editTime == null || editTime.equals("") || articleContent.text().contains("您要查看本帖隐藏内容请")) {
			System.out.println("不够权限，手动摘取");
			manmalArticle(doc.baseUri());
			return null;
		}
		editTime = editTime.substring(editTime.length() - 17, editTime.length() - 3) + ":00";
		artcleMap.put("created", editTime);
		editObjs.remove();

		// 替换所有超链接为空事件
		articleContent.select("a[href]").attr("href", "javascript:void()");
		// 替换图片
		Elements imgObjs = articleContent.select("img[src]");
		for (Element img : imgObjs) {
			img.attr("src", getNewImgUrl(img.attr("abs:src")));
		}

		artcleMap.put("content", articleContent.toString());
		artcleMap.put("tags", "信用卡," + getTag(articleContent.toString()));

		saveHtmFile(articleContent.toString());
		System.out.println(articleContent);

		return artcleMap;
	}

	void saveHtmFile(String text) throws IOException {
		String fileName =  UUID.randomUUID()+ "12.html";

		FileWriter fw = new FileWriter(new File(newPicDir + fileName));
		fw.write(text);
		fw.close();
	}

	String getNewImgUrl(String imgUrlStr) throws IOException {
		System.out.println("保存图片："+imgUrlStr);
		String fileName = UUID.randomUUID() + ".png";// 暂时写死PNG
		// jpg, jpeg, gif, png 
		if(imgUrlStr.endsWith(".jpg")){
			fileName = fileName.substring(0,fileName.length()-4) + ".jpg";
		}else if(imgUrlStr.endsWith(".jpeg")){
			fileName = fileName.substring(0,fileName.length()-4) + ".jpeg";
		}else if(imgUrlStr.endsWith(".gif")){
			fileName = fileName.substring(0,fileName.length()-4) + ".gif";
		}
		

		URL imgUrl = new URL(imgUrlStr);

		URLConnection uc = imgUrl.openConnection();
		InputStream is = uc.getInputStream();
		OutputStream os = new FileOutputStream(new File(newPicDir + fileName));
		IOUtils.copy(is, os);

		String webApp = newPicDir.substring(newPicDir.lastIndexOf("/webapps/"), newPicDir.length()).replaceAll("/webapps", "");
		return webApp + fileName;
	}
	
	/**
	 * 检查该文章是否已读取
	 * 
	 * @param tid
	 * @return
	 */
	Boolean isNewArticle(String tid) {
		String fileName = "credit51.txt";

		try {
			File f = new File(tysSpiderLogDir + fileName);
			FileUtils.forceMkdir(f);
			if (FileUtils.readFileToString(f).contains(tid + ";")) {

			} else {
				FileUtils.writeStringToFile(f, tid + ";", true);
				return true;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}

	/**
	 * 记录需要手动的文章URL
	 * 
	 * @param url
	 * @return
	 */
	Boolean manmalArticle(String url) {
		String fileName = "credit51_err.txt";

		try {
			File f = new File(tysSpiderLogDir + fileName);
			FileUtils.forceMkdir(f);
			FileUtils.writeStringToFile(f, url + "\n", true);
		} catch (IOException e) {
		//	e.printStackTrace();
		}
		return false;
	}

	String getArcticleId(String url) {
		// http://bbs.51credit.com/forum.php?mod=viewthread&tid=1037839&extra=page%3D1%26orderby%3Dheats
		String[] pArr = url.split("&");
		for (String p : pArr) {
			if (p.startsWith("tid=")) {
				return p.replaceAll("tid=", "");
			}
		}
		return "-1";
	}

	String getTag(String article) {
		String[] tagArr = tags.split(",");
		String tmpTag = "";
		for (String t : tagArr) {
			if (article.contains(t)) {
				tmpTag += t + ",";
			}
		}
		
		for(String tg:cur_tag){
			if(!tmpTag.contains(tg)){
				tmpTag+=tg + ",";
			}
		}
		return tmpTag;
	}

}
